! rm -rf Datasets
# Standard Imports
import os
# Third-Party Imports
import tensorflow as tf
import tensorflow_addons as tfa
from tensorflow.keras.utils import image_dataset_from_directory
import keras_tuner as kt
import plotly.express as px
import plotly.graph_objects as go
from plotly.subplots import make_subplots
import plotly.figure_factory as ff
import plotly.offline as pyo
pyo.init_notebook_mode()
import numpy as np
import pandas as pd
from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay
import splitfolders
Requirement already satisfied: tensorflow-addons in /Users/bhekimaenetja/.local/share/virtualenvs/small-projects-ai-NRjJWIjk/lib/python3.9/site-packages (0.19.0) Requirement already satisfied: typeguard>=2.7 in /Users/bhekimaenetja/.local/share/virtualenvs/small-projects-ai-NRjJWIjk/lib/python3.9/site-packages (from tensorflow-addons) (3.0.1) Requirement already satisfied: packaging in /Users/bhekimaenetja/.local/share/virtualenvs/small-projects-ai-NRjJWIjk/lib/python3.9/site-packages (from tensorflow-addons) (23.0) Requirement already satisfied: typing-extensions>=4.4.0 in /Users/bhekimaenetja/.local/share/virtualenvs/small-projects-ai-NRjJWIjk/lib/python3.9/site-packages (from typeguard>=2.7->tensorflow-addons) (4.5.0) Requirement already satisfied: importlib-metadata>=3.6 in /Users/bhekimaenetja/.local/share/virtualenvs/small-projects-ai-NRjJWIjk/lib/python3.9/site-packages (from typeguard>=2.7->tensorflow-addons) (6.1.0) Requirement already satisfied: zipp>=0.5 in /Users/bhekimaenetja/.local/share/virtualenvs/small-projects-ai-NRjJWIjk/lib/python3.9/site-packages (from importlib-metadata>=3.6->typeguard>=2.7->tensorflow-addons) (3.15.0)
# Plotting functions
def plot_data(x=None, y=None, z=None, title="", x_label="", y_label="", name="", mode="markers", text="", **traces):
fig = go.Figure(layout={
"title": title,
"xaxis": {"title": x_label},
"yaxis": {"title": y_label}
})
if z is None:
data = go.Scatter(
x=x,
y=y,
mode=mode,
name=name,
text=text
)
else:
data = go.Scatter3d(
x=x,
y=y,
z=z,
mode=mode,
name=name,
text=text
)
if x is not None and y is not None:
fig.add_trace(data)
for t in traces:
fig.add_trace(traces[t])
return fig
def plot_bar_data(*bars, x=None, title="", x_label="", y_label=""):
fig = go.Figure(
layout={
"title": title,
"xaxis": {"title": x_label},
"yaxis": {"title": y_label},
"barmode": "group"
}, data=[
go.Bar(name=f"{bar[0]}", x=x, y=bar[1])
for bar in bars
])
return fig
def create_trace(x=None, y=None, z=None, name="", mode="lines", text="", marker_size=None):
if z is None:
trace = go.Scatter(
x=x,
y=y,
mode=mode,
name=name,
text=text,
marker=dict(size=marker_size)
)
else:
trace = go.Scatter3d(
x=x,
y=y,
z=z,
mode=mode,
name=name,
text=text,
marker=dict(size=marker_size)
)
return trace
def create_confusion_matrix(dataset, preds):
preds = 1 * (preds >= 0.5).flatten()
labels = []
for x, y in dataset.map(lambda x, y: (x, y)):
labels.append(y.numpy())
labels = np.concatenate(labels)
basic_cm = confusion_matrix(labels, preds)
return basic_cm
def plot_confusion_matrix(cm, x=[0,1], y=[0,1], title="", x_label="Predicted Label", y_label="True Label"):
fig = ff.create_annotated_heatmap(
cm,
x=x,
y=y,
annotation_text=cm,
colorscale='Viridis',
showscale=True,
)
fig.update_layout({
"title": title,
"xaxis": {"title": x_label, "side": "bottom"},
"yaxis": {"title": y_label, "autorange": "reversed"},
"coloraxis": { "colorbar": {"orientation": "h"}}
})
return fig
def plot_confusion_matrix_2(cm, display_labels=["Evil", "Good"]):
return ConfusionMatrixDisplay(
confusion_matrix=cm, display_labels=display_labels
)
return cm
def plot_metric_table(metrics):
# Initialise figure
fig = go.Figure()
# Create table
headers = dict(values=["", "Training Set", "Validation Set", "Test Set"])
first_column = [
"F1 Score",
"Accuracy",
"Precision",
"Recall",
"True Postives",
"False Positives",
"True Negatives",
"False Negatives",
"Loss"
]
cells = [first_column] + [[
round(metrics[ds]["f1_score"], 4),
round(metrics[ds]["accuracy"], 4),
round(metrics[ds]["precision"], 4),
round(metrics[ds]["recall"], 4),
metrics[ds]["tp"],
metrics[ds]["fp"],
metrics[ds]["tn"],
metrics[ds]["fn"],
round(metrics[ds]["loss"], 4)
] for ds in ("train_set", "val_set", "test_set")]
data = go.Table(
header=headers,
cells=dict(values=cells)
)
fig.add_trace(data)
return fig
def train_plot_collection(training_plots, metric_table, title=""):
# Initialise make_subplots object
fig = make_subplots(
rows=4,
cols=2,
subplot_titles=[
"Accuracy",
"Loss",
"Precision",
"Recall",
"F1_score",
"Key Metrics",
],
specs=[
[{"type": "xy"}, {"type": "xy"}],
[{"type": "xy"}, {"type": "xy"}],
[{"type": "xy", "colspan": 2}, None],
[{"type": "table", "colspan": 2}, None],
]
)
fig.update_layout({
"title": title,
"height": 1500,
})
# Add traces
for i in range(len(training_plots["accuracy"].data)):
fig.add_trace(training_plots["accuracy"].data[i], row=1, col=1)
for i in range(len(training_plots["loss"].data)):
fig.add_trace(training_plots["loss"].data[i], row=1, col=2)
for i in range(len(training_plots["precision"].data)):
fig.add_trace(training_plots["precision"].data[i], row=2, col=1)
for i in range(len(training_plots["recall"].data)):
fig.add_trace(training_plots["recall"].data[i], row=2, col=2)
for i in range(len(training_plots["f1_score"].data)):
fig.add_trace(training_plots["f1_score"].data[i], row=3, col=1)
fig.add_trace(metric_table.data[0], row=4, col=1)
# Update axes
fig.update_xaxes(title_text="Epochs", row=1, col=1)
fig.update_xaxes(title_text="Epochs", row=1, col=2)
fig.update_xaxes(title_text="Epochs", row=2, col=1)
fig.update_xaxes(title_text="Epochs", row=2, col=2)
fig.update_xaxes(title_text="Epochs", row=3, col=1)
return fig
def test_plot_collection(metric_plot, true_false_plot, title=""):
# Initialise make_subplots object
fig = make_subplots(
rows=2,
cols=1,
shared_xaxes=True,
vertical_spacing=0.03,
)
fig.update_layout({
"title": title,
"height": 1000,
})
# Add Traces
for i in range(len(metric_plot.data)):
fig.add_trace(metric_plot.data[i], row=1, col=1)
for i in range(len(true_false_plot.data)):
fig.add_trace(true_false_plot.data[i], row=2, col=1)
return fig
# Global variables
train_dir = "Datasets/train" # repo for training set
val_dir = "Datasets/val" # repo for validation set
test_dir = "Datasets/test" # repo for test set
img_h = 140 # image height
img_w = 140 # image width
batch_size = 32
label_map = {0: "evil", 1: "good"} # "evil" is the negative class; "good" is the positive class
input_shape = (img_h, img_w, 3)
dataset_split_ratio = (0.7, 0.15, 0.15)
seed = 312
# Splitting dataset into training, validation and testing sets
splitfolders.ratio(
"Images",
output="Datasets",
ratio=dataset_split_ratio,
seed=seed,
)
# source: https://github.com/jfilter/split-folders
Copying files: 597 files [00:00, 1169.27 files/s]
# Load training set
train_ds = image_dataset_from_directory(
train_dir,
validation_split=0,
seed=seed,
image_size=(img_h, img_w),
batch_size=batch_size
)
# Load validation set
val_ds = image_dataset_from_directory(
val_dir,
validation_split=0,
seed=seed,
image_size=(img_h, img_w),
batch_size=batch_size
)
# Load testing set
test_ds = image_dataset_from_directory(
test_dir,
validation_split=0,
seed=seed,
image_size=(img_h, img_w),
batch_size=batch_size
)
Found 417 files belonging to 2 classes. Found 89 files belonging to 2 classes. Found 91 files belonging to 2 classes.
y1 = np.concatenate([y for x, y in train_ds], axis=0)
y2 = np.concatenate([y for x, y in val_ds], axis=0)
y3 = np.concatenate([y for x, y in test_ds], axis=0)
class_totals = np.bincount(y1) + np.bincount(y2) + np.bincount(y3)
neg, pos = class_totals[0], class_totals[1]
total = neg + pos
evil_weight = (1 / neg) * (total / 2.0)
good_weight = (1 / pos) * (total / 2.0)
class_weights = {0: evil_weight, 1: good_weight}
class_weights
# source: https://www.tensorflow.org/tutorials/structured_data/imbalanced_data#calculate_class_weights
{0: 1.515228426395939, 1: 0.74625}
for images, labels in train_ds.take(1):
for i in range(batch_size // 16):
print(label_map[labels[i].numpy()])
go.Figure(go.Image(z=images[i].numpy())).show()
good
evil
def perceptron(input_shape, num_outputs=1):
model = tf.keras.Sequential([
tf.keras.layers.Rescaling(1./255),
tf.keras.layers.Flatten(input_shape=input_shape),
tf.keras.layers.Dense(num_outputs, activation='sigmoid')
])
return model
def test_cnn_1(input_shape, num_outputs=1):
model = tf.keras.Sequential([
tf.keras.layers.Rescaling(1./255),
tf.keras.layers.BatchNormalization(),
tf.keras.layers.Conv2D(16, (3, 3), activation="relu", input_shape=input_shape, kernel_regularizer="l2"),
tf.keras.layers.MaxPooling2D((2, 2)),
tf.keras.layers.Dropout(0.5),
tf.keras.layers.Flatten(),
tf.keras.layers.BatchNormalization(),
tf.keras.layers.Dense(num_outputs, activation="sigmoid")
])
return model
# source: https://www.tensorflow.org/tutorials/images/cnn
def test_cnn_2(input_shape, num_outputs=1):
model = tf.keras.Sequential([
tf.keras.layers.Rescaling(1./255),
tf.keras.layers.BatchNormalization(),
tf.keras.layers.Conv2D(16, (3, 3), activation="relu", input_shape=input_shape, kernel_regularizer="l2"),
tf.keras.layers.MaxPooling2D((2, 2)),
tf.keras.layers.Dropout(0.5),
tf.keras.layers.Conv2D(32, (3, 3), activation="relu", kernel_regularizer="l2"),
tf.keras.layers.MaxPooling2D((2, 2)),
tf.keras.layers.Dropout(0.5),
tf.keras.layers.Flatten(),
tf.keras.layers.BatchNormalization(),
tf.keras.layers.Dense(num_outputs, activation='sigmoid')
])
return model
# source: https://www.tensorflow.org/tutorials/images/cnn
def test_cnn_3(input_shape, num_outputs=1):
model = tf.keras.Sequential([
tf.keras.layers.Rescaling(1./255),
tf.keras.layers.BatchNormalization(),
tf.keras.layers.Conv2D(16, (3, 3), activation="relu", input_shape=input_shape, kernel_regularizer="l2"),
tf.keras.layers.MaxPooling2D((2, 2)),
tf.keras.layers.Dropout(0.5),
tf.keras.layers.Conv2D(32, (3, 3), activation="relu", kernel_regularizer="l2"),
tf.keras.layers.MaxPooling2D((2, 2)),
tf.keras.layers.Dropout(0.5),
tf.keras.layers.Conv2D(64, (3, 3), activation="relu", kernel_regularizer="l2"),
tf.keras.layers.MaxPooling2D((2, 2)),
tf.keras.layers.Dropout(0.5),
tf.keras.layers.Flatten(),
tf.keras.layers.BatchNormalization(),
tf.keras.layers.Dense(1, activation="sigmoid")
])
return model
# source: https://www.tensorflow.org/tutorials/images/cnn
def get_model(model_name, input_shape):
models = {
"perceptron": perceptron,
"test_cnn_1": test_cnn_1,
"test_cnn_2": test_cnn_2,
"test_cnn_3": test_cnn_3,
}
return models[model_name](input_shape)
def get_optimiser(opt_name, l_rate, epsilon):
optimisers = {
"adam": tf.keras.optimizers.Adam,
"sgd": tf.keras.optimizers.experimental.SGD,
"adadelta": tf.keras.optimizers.experimental.Adadelta,
"adagrad": tf.keras.optimizers.experimental.Adagrad,
"adamax": tf.keras.optimizers.experimental.Adamax,
"nadam": tf.keras.optimizers.experimental.Nadam,
"rmsprop": tf.keras.optimizers.experimental.RMSprop
}
if opt_name == "sgd": return optimisers[opt_name](learning_rate=l_rate)
return optimisers[opt_name](learning_rate=l_rate, epsilon=epsilon)
def get_train_performance_plots(history, epochs, loss_metrics, acc_metrics, prec_metrics, rec_metrics, f1_metrics):
x = np.arange(1, epochs + 1)
loss_traces = {
m: create_trace(x, history[m], name=m)
for m in loss_metrics
}
f1_traces = {
m: create_trace(x, history[m], name=m)
for m in f1_metrics
}
accuracy_traces = {
m: create_trace(x, history[m], name=m)
for m in acc_metrics
}
precision_traces = {
m: create_trace(x, history[m], name=m)
for m in prec_metrics
}
recall_traces = {
m: create_trace(x, history[m], name=m)
for m in rec_metrics
}
train_acc_plot = plot_data(
title="Training Accuracy",
x_label="Epochs",
y_label="Accuracy",
**accuracy_traces
)
train_loss_plot = plot_data(
title="Training Loss",
x_label="Epochs",
y_label="Loss",
**loss_traces
)
train_prec_plot = plot_data(
title="Training Precision",
x_label="Epochs",
y_label="Precision",
**precision_traces
)
train_rec_plot = plot_data(
title="Training Recall",
x_label="Epochs",
y_label="Recall",
**recall_traces
)
train_f1_plot = plot_data(
title="Training F1 Score",
x_label="Epochs",
y_label="F1 Score",
**f1_traces
)
return train_acc_plot, train_loss_plot, train_prec_plot, train_rec_plot, train_f1_plot
def get_confusion_matrix(cm):
return plot_confusion_matrix(cm, x=["Evil", "Good"], y=["Evil", "Good"])
def get_confusion_matrix_2(cm, display_labels=["Evil", "Good"]):
return plot_confusion_matrix_2(cm, display_labels)
def get_metric_table(metrics):
return plot_metric_table(metrics)
def run_model(
model_name,
train_set,
val_set,
test_set,
input_shape=(140,140,3),
epochs=10,
opt_name="sgd",
l_rate=0.001,
epsilon=1e-7,
class_weight=None,
verbosity="auto",
):
# Choose and initialise model
model = get_model(model_name, input_shape)
# Choose and initialise optimiser
optimiser = get_optimiser(opt_name, l_rate, epsilon)
# Compile model
model.compile(
optimizer=optimiser,
loss="binary_crossentropy",
metrics=[
tfa.metrics.F1Score(
num_classes=2,
threshold=0.5,
average="micro"
),
"accuracy",
tf.keras.metrics.Precision(),
tf.keras.metrics.Recall(),
]
)
# Fit model to data
history = model.fit(
train_set,
epochs=epochs,
validation_data=val_set,
shuffle=True,
class_weight=class_weight,
verbose=verbosity,
).history
keys = list(history.keys())
for k in keys:
if "val_accuracy" in k and k != "val_accuracy":
history["val_precision"] = history[k]
elif "val_f1_score" in k and k != "val_f1_score":
history["val_recall"] = history[k]
elif "accuracy" in k and k != "accuracy":
history["precision"] = history[k]
elif "f1_score" in k and k != "f1_score":
history["recall"] = history[k]
elif "val_precision" in k and k != "val_precision":
history["val_precision"] = history[k]
elif "val_recall" in k and k != "val_recall":
history["val_recall"] = history[k]
elif "precision" in k and k != "precision":
history["precision"] = history[k]
elif "recall" in k and k != "recall":
history["recall"] = history[k]
# Evaluate model on train, validation and test sets
metric_names = ["loss", "f1_score", "accuracy", "precision", "recall"]
train_metrics = model.evaluate(train_set)
val_metrics = model.evaluate(val_set)
test_metrics = model.evaluate(test_set)
metrics = {
"train_set": {
name: metric
for name, metric in zip(metric_names, train_metrics)
},
"val_set": {
name: metric
for name, metric in zip(metric_names, val_metrics)
},
"test_set": {
name: metric
for name, metric in zip(metric_names, test_metrics)
},
}
# Generate predictions on the test set
preds = {
"train_set": model.predict(train_set),
"val_set": model.predict(val_set),
"test_set": model.predict(test_set),
}
# Build training and loss graphs
loss_metrics = ["loss", "val_loss"]
accuracy_metrics = ["accuracy", "val_accuracy"]
prec_metrics = ["precision", "val_precision"]
rec_metrics = ["recall", "val_recall"]
f1_metrics = ["f1_score", "val_f1_score"]
train_acc_plot, train_loss_plot, train_prec_plot, train_rec_plot, train_f1_plot = get_train_performance_plots(
history,
epochs,
loss_metrics,
accuracy_metrics,
prec_metrics,
rec_metrics,
f1_metrics,
)
training_plots = {
"accuracy": train_acc_plot,
"loss": train_loss_plot,
"precision": train_prec_plot,
"recall": train_rec_plot,
"f1_score": train_f1_plot
}
# Get cofusion matrices for train, validation and test sets
basic_cms = {
"train_set": create_confusion_matrix(train_set, preds["train_set"]),
"val_set": create_confusion_matrix(val_set, preds["val_set"]),
"test_set": create_confusion_matrix(test_set, preds["test_set"]),
}
confusion_matrices = {
"train_set": get_confusion_matrix(basic_cms["train_set"]),
"val_set": get_confusion_matrix(basic_cms["val_set"]),
"test_set": get_confusion_matrix(basic_cms["test_set"])
}
# Append new metrics from confusion matrices
for d_set in ("train_set", "val_set", "test_set"):
metrics[d_set]["tp"] = basic_cms[d_set][1][1]
metrics[d_set]["fp"] = basic_cms[d_set][0][1]
metrics[d_set]["fn"] = basic_cms[d_set][1][0]
metrics[d_set]["tn"] = basic_cms[d_set][0][0]
# Create metric table
metric_table = get_metric_table(metrics)
return model, metrics, metric_table, preds, training_plots, basic_cms, confusion_matrices
model, metrics, metric_table, preds, training_plots, basic_cms, confusion_matrices = run_model(
model_name="test_cnn_1",
train_set=train_ds,
val_set=val_ds,
test_set=test_ds,
input_shape=input_shape,
epochs=10,
l_rate=0.01,
opt_name="sgd",
class_weight=class_weights,
)
Epoch 1/10 WARNING:tensorflow:From /Users/bhekimaenetja/.local/share/virtualenvs/small-projects-ai-NRjJWIjk/lib/python3.9/site-packages/tensorflow/python/autograph/pyct/static_analysis/liveness.py:83: Analyzer.lamba_check (from tensorflow.python.autograph.pyct.static_analysis.liveness) is deprecated and will be removed after 2023-09-23. Instructions for updating: Lambda fuctions will be no more assumed to be used in the statement where they are used, or at least in the same block. https://github.com/tensorflow/tensorflow/issues/56089 14/14 [==============================] - 5s 250ms/step - loss: 1.5125 - f1_score: 0.5904 - accuracy: 0.5108 - precision: 0.6743 - recall: 0.5250 - val_loss: 0.7193 - val_f1_score: 0.7973 - val_accuracy: 0.6629 - val_precision: 0.6705 - val_recall: 0.9833 Epoch 2/10 14/14 [==============================] - 3s 231ms/step - loss: 0.9615 - f1_score: 0.6937 - accuracy: 0.6379 - precision: 0.8028 - recall: 0.6107 - val_loss: 0.8414 - val_f1_score: 0.8108 - val_accuracy: 0.6854 - val_precision: 0.6818 - val_recall: 1.0000 Epoch 3/10 14/14 [==============================] - 3s 216ms/step - loss: 0.9126 - f1_score: 0.7076 - accuracy: 0.6571 - precision: 0.8278 - recall: 0.6179 - val_loss: 1.0567 - val_f1_score: 0.5714 - val_accuracy: 0.4944 - val_precision: 0.6667 - val_recall: 0.5000 Epoch 4/10 14/14 [==============================] - 3s 227ms/step - loss: 0.6126 - f1_score: 0.7960 - accuracy: 0.7554 - precision: 0.9045 - recall: 0.7107 - val_loss: 0.6394 - val_f1_score: 0.8088 - val_accuracy: 0.7079 - val_precision: 0.7237 - val_recall: 0.9167 Epoch 5/10 14/14 [==============================] - 3s 230ms/step - loss: 0.5736 - f1_score: 0.7921 - accuracy: 0.7482 - precision: 0.8889 - recall: 0.7143 - val_loss: 0.7195 - val_f1_score: 0.7727 - val_accuracy: 0.6629 - val_precision: 0.7083 - val_recall: 0.8500 Epoch 6/10 14/14 [==============================] - 3s 234ms/step - loss: 0.3942 - f1_score: 0.8413 - accuracy: 0.8082 - precision: 0.9464 - recall: 0.7571 - val_loss: 0.7452 - val_f1_score: 0.7538 - val_accuracy: 0.6404 - val_precision: 0.7000 - val_recall: 0.8167 Epoch 7/10 14/14 [==============================] - 3s 232ms/step - loss: 0.3944 - f1_score: 0.8599 - accuracy: 0.8273 - precision: 0.9444 - recall: 0.7893 - val_loss: 0.7594 - val_f1_score: 0.7360 - val_accuracy: 0.6292 - val_precision: 0.7077 - val_recall: 0.7667 Epoch 8/10 14/14 [==============================] - 3s 224ms/step - loss: 0.4234 - f1_score: 0.8571 - accuracy: 0.8249 - precision: 0.9481 - recall: 0.7821 - val_loss: 0.8510 - val_f1_score: 0.7143 - val_accuracy: 0.5955 - val_precision: 0.6818 - val_recall: 0.7500 Epoch 9/10 14/14 [==============================] - 3s 227ms/step - loss: 0.4464 - f1_score: 0.8642 - accuracy: 0.8297 - precision: 0.9300 - recall: 0.8071 - val_loss: 0.8541 - val_f1_score: 0.7500 - val_accuracy: 0.6404 - val_precision: 0.7059 - val_recall: 0.8000 Epoch 10/10 14/14 [==============================] - 3s 218ms/step - loss: 0.3185 - f1_score: 0.9015 - accuracy: 0.8753 - precision: 0.9597 - recall: 0.8500 - val_loss: 0.8353 - val_f1_score: 0.7692 - val_accuracy: 0.6629 - val_precision: 0.7143 - val_recall: 0.8333 14/14 [==============================] - 1s 38ms/step - loss: 0.2060 - f1_score: 0.9704 - accuracy: 0.9592 - precision: 0.9458 - recall: 0.9964 3/3 [==============================] - 0s 42ms/step - loss: 0.8353 - f1_score: 0.7692 - accuracy: 0.6629 - precision: 0.7143 - recall: 0.8333 3/3 [==============================] - 0s 40ms/step - loss: 0.8737 - f1_score: 0.7500 - accuracy: 0.6264 - precision: 0.6711 - recall: 0.8500 14/14 [==============================] - 1s 39ms/step 3/3 [==============================] - 0s 37ms/step 3/3 [==============================] - 0s 68ms/step
train_plot_collection(training_plots, metric_table, "Training Results for Test Model 1")
# Get predictions
preds["test_set"]
array([[0.81288254],
[0.611088 ],
[0.8975335 ],
[0.7974999 ],
[0.8789054 ],
[0.95714366],
[0.8393403 ],
[0.89898956],
[0.03007286],
[0.2317079 ],
[0.9409177 ],
[0.31317553],
[0.84566253],
[0.8655793 ],
[0.94851184],
[0.88803065],
[0.90942335],
[0.9584492 ],
[0.96664816],
[0.7384149 ],
[0.57421404],
[0.9040143 ],
[0.30616298],
[0.8886775 ],
[0.96876436],
[0.99736655],
[0.8795997 ],
[0.9006875 ],
[0.7856223 ],
[0.47414273],
[0.96970284],
[0.84734094],
[0.4006444 ],
[0.9896213 ],
[0.88879615],
[0.1733048 ],
[0.8556108 ],
[0.7576365 ],
[0.9480191 ],
[0.9472297 ],
[0.58313245],
[0.83794457],
[0.98230207],
[0.6320949 ],
[0.38295907],
[0.7878443 ],
[0.7271949 ],
[0.9726155 ],
[0.7567242 ],
[0.98197377],
[0.86900985],
[0.88539845],
[0.47246763],
[0.6321067 ],
[0.3614071 ],
[0.8880345 ],
[0.7326922 ],
[0.85527843],
[0.02947607],
[0.87678087],
[0.9179377 ],
[0.8151784 ],
[0.64918774],
[0.94750154],
[0.7668669 ],
[0.9664161 ],
[0.74536943],
[0.2750769 ],
[0.5977465 ],
[0.43228367],
[0.9477465 ],
[0.92996174],
[0.81136954],
[0.671169 ],
[0.73293364],
[0.7469404 ],
[0.74196726],
[0.6985409 ],
[0.69045407],
[0.795322 ],
[0.9368606 ],
[0.59249955],
[0.96143323],
[0.809855 ],
[0.9753247 ],
[0.7798728 ],
[0.85444766],
[0.24379267],
[0.6337953 ],
[0.95332026],
[0.4746781 ]], dtype=float32)
# Confusion matrix to tell us the number of true positives, false negatives etc.
confusion_matrices["train_set"]
confusion_matrices["val_set"]
confusion_matrices["test_set"]
The hyperparameters that we'll look at are: # of epochs, learning rate, optimiser and epsilon.
def get_test_plots(x, acc, prec, rec, f1, tp, tn, fp, fn):
true_false_plot = plot_bar_data(
tp,
tn,
fp,
fn,
x=x,
x_label="Epoch Number",
)
metric_plot = plot_bar_data(
acc,
prec,
rec,
f1,
x=x,
title="Test Bar Chart 2",
x_label="Parameter",
)
return true_false_plot, metric_plot
def tune_hyper_params(model_name, optimiser_range):
def model_builder(hp):
model = get_model(model_name, input_shape=(140, 140, 3))
hp_l_rate = hp.Float("learning_rate", min_value=0, max_value=1, step=1e-08)
hp_opt_name = hp.Choice("optimiser", values=optimiser_range)
hp_epsilon = hp.Float("epsilon", min_value=0, max_value=1, step=1e-08)
optimiser = get_optimiser(hp_opt_name, hp_l_rate, hp_epsilon)
model.compile(
optimizer=optimiser,
loss="binary_crossentropy",
metrics=[
"accuracy",
tf.keras.metrics.Precision(),
tf.keras.metrics.Recall(),
]
)
return model
return model_builder
def get_tuner(model_builder, tuner_name):
if tuner_name == "bayesian":
tuner = kt.BayesianOptimization(
model_builder,
max_trials=5,
objective="val_accuracy",
directory="",
overwrite=True,
)
elif tuner_name == "random":
tuner = kt.RandomSearch(
model_builder,
max_trials=5,
objective="val_accuracy",
directory="",
overwrite=True,
)
return tuner
def get_best_hyperparams(model_name, train_set, val_set, optimiser_range, tuner_name, class_weights=class_weights):
print(f"Class weight is: {class_weights}")
model_builder = tune_hyper_params(model_name, optimiser_range)
tuner = get_tuner(model_builder, tuner_name)
tuner.search(
train_set,
epochs=5,
validation_data=val_set,
shuffle=True,
class_weight=class_weights,
verbose=1,
)
best_hps = tuner.get_best_hyperparameters(num_trials=50)[0]
return {
"optimiser": best_hps.get("optimiser"),
"l_rate": best_hps.get("learning_rate"),
"epsilon": best_hps.get("epsilon"),
}
# source: https://keras.io/api/keras_tuner/
def compare_configs(test_params, model_name, train_set, val_set, test_set, control_params={}):
# Check if all param ranges are of equal length
vals = list(test_params.values())
val_len = len(vals[0])
for v in vals[1:]:
if len(v) != val_len:
raise Exception("Error: Param ranges are not all the same!")
# Initialise parameters for running model
default_params = {
"model_name": model_name,
"train_set": train_set,
"val_set": val_set,
"test_set": test_set,
"epochs": 10,
"opt_name": "sgd",
"l_rate": 0.001,
"epsilon": 1e-7,
"class_weight": None,
"verbosity": 0,
}
default_params.update(control_params)
# Initialise X-axis
param_names = list(test_params.keys())
x = []
for i in range(val_len):
x_string = ""
for n in param_names:
x_string += f" {n} = {test_params[n][i]};"
x.append(x_string[:-1].strip())
# Iterate through parameters
acc = ("Accuracy", [])
prec = ("Precision", [])
rec = ("Recall", [])
f1 = ("F1 Score", [])
tp = ("True Positives", [])
tn = ("True Negatives", [])
fp = ("False Posivites", [])
fn = ("False Negatives", [])
for i in range(val_len):
params_copy = default_params.copy()
test_params_i = {n: test_params[n][i] for n in param_names}
params_copy.update(test_params_i)
model, metrics, metric_table, _, training_plots, _, _ = run_model(**params_copy)
train_results = train_plot_collection(
training_plots,
metric_table,
f"Results ({x[i]})"
)
acc[1].append(metrics["test_set"]["accuracy"])
prec[1].append(metrics["test_set"]["precision"])
rec[1].append(metrics["test_set"]["recall"])
f1[1].append(metrics["test_set"]["f1_score"])
tp[1].append(metrics["test_set"]["tp"])
tn[1].append(metrics["test_set"]["tn"])
fp[1].append(metrics["test_set"]["fp"])
fn[1].append(metrics["test_set"]["fn"])
train_results.show()
# Get final result plots
true_false_plot, metric_plot = get_test_plots(x,acc,prec,rec,f1,tp,tn,fp,fn)
results = test_plot_collection(
true_false_plot,
metric_plot,
f"Test set results when adjusting: {', '.join([n for n in param_names])}"
)
return results
optimiser_range = ["sgd", "adam", "adadelta", "adagrad", "adamax", "nadam", "rmsprop"]
hp_search_methods = ["random", "bayesian"]
model_hps = dict()
model_name = "perceptron"
best_hps = {
method: get_best_hyperparams(model_name, train_ds, val_ds, optimiser_range, method)
for method in hp_search_methods
}
best_hps
Trial 5 Complete [00h 00m 02s] val_accuracy: 0.6741573214530945 Best val_accuracy So Far: 0.6741573214530945 Total elapsed time: 00h 00m 12s INFO:tensorflow:Oracle triggered exit
{'random': {'optimiser': 'rmsprop',
'l_rate': 0.62089814,
'epsilon': 0.61781427},
'bayesian': {'optimiser': 'adagrad',
'l_rate': 0.022246600000000002,
'epsilon': 0.7611846600000001}}
# parameters that we want to iterate through
hp_matrix = [
x
for x in set(
(
best_hps[hp]["optimiser"],
best_hps[hp]["l_rate"],
best_hps[hp]["epsilon"],
)
for hp in hp_search_methods
)
]
test_params = {
"opt_name": [hp[0] for hp in hp_matrix],
"l_rate": [hp[1] for hp in hp_matrix],
"epsilon": [hp[2] for hp in hp_matrix],
}
# parameters that differ from the default values but
# that we want to keep constant through each iteration
control_params = {
"model_name": model_name,
}
compare_configs(
test_params,
model_name,
train_ds,
val_ds,
test_ds,
control_params,
)
14/14 [==============================] - 0s 5ms/step - loss: 27.4598 - f1_score: 0.8034 - accuracy: 0.6715 - precision_1: 0.6715 - recall_1: 1.0000 3/3 [==============================] - 0s 7ms/step - loss: 25.7476 - f1_score: 0.8054 - accuracy: 0.6742 - precision_1: 0.6742 - recall_1: 1.0000 3/3 [==============================] - 0s 7ms/step - loss: 30.5270 - f1_score: 0.7947 - accuracy: 0.6593 - precision_1: 0.6593 - recall_1: 1.0000 14/14 [==============================] - 0s 5ms/step 3/3 [==============================] - 0s 4ms/step 3/3 [==============================] - 0s 5ms/step
14/14 [==============================] - 0s 6ms/step - loss: 663.3700 - f1_score: 0.8046 - accuracy: 0.6739 - precision_2: 0.6731 - recall_2: 1.0000 3/3 [==============================] - 0s 7ms/step - loss: 709.2269 - f1_score: 0.8054 - accuracy: 0.6742 - precision_2: 0.6742 - recall_2: 1.0000 3/3 [==============================] - 0s 8ms/step - loss: 905.7441 - f1_score: 0.7947 - accuracy: 0.6593 - precision_2: 0.6593 - recall_2: 1.0000 14/14 [==============================] - 0s 5ms/step 3/3 [==============================] - 0s 5ms/step 3/3 [==============================] - 0s 4ms/step
best_optimiser = "adagrad"
best_l_rate = 0.022246600000000002
best_epsilon = 0.7611846600000001
# parameters that we want to iterate through
test_params = {
"epochs": [50]
}
# parameters that differ from the default values but
# that we want to keep constant through each iteration
# of the test parameters
control_params = {
"model_name": model_name,
"opt_name": best_optimiser,
"l_rate": best_l_rate,
"epsilon": best_epsilon,
}
compare_configs(
test_params,
model_name,
train_ds,
val_ds,
test_ds,
control_params,
)
14/14 [==============================] - 0s 5ms/step - loss: 0.0410 - f1_score: 0.9928 - accuracy: 0.9904 - precision_3: 1.0000 - recall_3: 0.9857 3/3 [==============================] - 0s 6ms/step - loss: 2.1981 - f1_score: 0.6111 - accuracy: 0.5281 - precision_3: 0.6875 - recall_3: 0.5500 3/3 [==============================] - 0s 7ms/step - loss: 2.1148 - f1_score: 0.6949 - accuracy: 0.6044 - precision_3: 0.7069 - recall_3: 0.6833 14/14 [==============================] - 0s 5ms/step 3/3 [==============================] - 0s 4ms/step 3/3 [==============================] - 0s 4ms/step
Ideal epoch number: 20
model_hps[model_name] = {
"epochs": 20,
"l_rate": best_l_rate,
"epsilon": best_epsilon,
"optimiser": best_optimiser
}
model_hps
{'perceptron': {'epochs': 20,
'l_rate': 0.022246600000000002,
'epsilon': 0.7611846600000001,
'optimiser': 'adagrad'}}
model_name = "test_cnn_1"
best_hps = {
method: get_best_hyperparams(model_name, train_ds, val_ds, optimiser_range, method)
for method in hp_search_methods
}
best_hps
Trial 5 Complete [00h 00m 18s] val_accuracy: 0.6853932738304138 Best val_accuracy So Far: 0.7078651785850525 Total elapsed time: 00h 01m 30s INFO:tensorflow:Oracle triggered exit
{'random': {'optimiser': 'adam', 'l_rate': 0.50438527, 'epsilon': 0.05359655},
'bayesian': {'optimiser': 'adagrad',
'l_rate': 0.52687047,
'epsilon': 0.89892038}}
# parameters that we want to iterate through
hp_matrix = [
x
for x in set(
(
best_hps[hp]["optimiser"],
best_hps[hp]["l_rate"],
best_hps[hp]["epsilon"],
)
for hp in hp_search_methods
)
]
test_params = {
"opt_name": [hp[0] for hp in hp_matrix],
"l_rate": [hp[1] for hp in hp_matrix],
"epsilon": [hp[2] for hp in hp_matrix],
}
# parameters that differ from the default values but
# that we want to keep constant through each iteration
control_params = {
"model_name": model_name,
}
compare_configs(
test_params,
model_name,
train_ds,
val_ds,
test_ds,
control_params,
)
14/14 [==============================] - 1s 40ms/step - loss: 1005.7009 - f1_score: 0.8141 - accuracy: 0.6954 - precision_3: 0.6898 - recall_3: 0.9929 3/3 [==============================] - 0s 42ms/step - loss: 1355.4073 - f1_score: 0.7891 - accuracy: 0.6517 - precision_3: 0.6667 - recall_3: 0.9667 3/3 [==============================] - 0s 44ms/step - loss: 1336.3508 - f1_score: 0.7891 - accuracy: 0.6593 - precision_3: 0.6667 - recall_3: 0.9667 14/14 [==============================] - 1s 37ms/step 3/3 [==============================] - 0s 36ms/step 3/3 [==============================] - 0s 40ms/step
14/14 [==============================] - 1s 41ms/step - loss: 178.6911 - f1_score: 0.0000e+00 - accuracy: 0.3285 - precision_4: 0.0000e+00 - recall_4: 0.0000e+00 3/3 [==============================] - 0s 36ms/step - loss: 188.4628 - f1_score: 0.0000e+00 - accuracy: 0.3258 - precision_4: 0.0000e+00 - recall_4: 0.0000e+00 3/3 [==============================] - 0s 39ms/step - loss: 187.2834 - f1_score: 0.0000e+00 - accuracy: 0.3407 - precision_4: 0.0000e+00 - recall_4: 0.0000e+00 14/14 [==============================] - 1s 36ms/step 3/3 [==============================] - 0s 35ms/step 3/3 [==============================] - 0s 37ms/step
best_optimiser = "adam"
best_l_rate = 0.50438527
best_epsilon = 0.05359655
# parameters that we want to iterate through
test_params = {
"epochs": [50]
}
# parameters that differ from the default values but
# that we want to keep constant through each iteration
# of the test parameters
control_params = {
"model_name": model_name,
"opt_name": best_optimiser,
"l_rate": best_l_rate,
"epsilon": best_epsilon
}
compare_configs(
test_params,
model_name,
train_ds,
val_ds,
test_ds,
control_params,
)
14/14 [==============================] - 1s 50ms/step - loss: 218.3432 - f1_score: 0.8034 - accuracy: 0.6715 - precision_7: 0.6715 - recall_7: 1.0000 3/3 [==============================] - 0s 51ms/step - loss: 217.8226 - f1_score: 0.8054 - accuracy: 0.6742 - precision_7: 0.6742 - recall_7: 1.0000 3/3 [==============================] - 0s 45ms/step - loss: 226.5419 - f1_score: 0.7947 - accuracy: 0.6593 - precision_7: 0.6593 - recall_7: 1.0000 14/14 [==============================] - 1s 43ms/step 3/3 [==============================] - 0s 72ms/step 3/3 [==============================] - 0s 39ms/step
Ideal epoch number: 15
model_hps[model_name] = {
"epochs": 15,
"l_rate": best_l_rate,
"epsilon": best_epsilon,
"optimiser": best_optimiser
}
model_hps
{'perceptron': {'epochs': 20,
'l_rate': 0.022246600000000002,
'epsilon': 0.7611846600000001,
'optimiser': 'adagrad'},
'test_cnn_1': {'epochs': 15,
'l_rate': 0.50438527,
'epsilon': 0.05359655,
'optimiser': 'adam'}}
model_name = "test_cnn_2"
best_hps = {
method: get_best_hyperparams(model_name, train_ds, val_ds, optimiser_range, method)
for method in hp_search_methods
}
best_hps
Trial 5 Complete [00h 00m 21s] val_accuracy: 0.6741573214530945 Best val_accuracy So Far: 0.6741573214530945 Total elapsed time: 00h 01m 47s INFO:tensorflow:Oracle triggered exit
{'random': {'optimiser': 'adam',
'l_rate': 0.8048532700000001,
'epsilon': 0.91542115},
'bayesian': {'optimiser': 'sgd',
'l_rate': 0.5362420800000001,
'epsilon': 0.05256905}}
# parameters that we want to iterate through
hp_matrix = [
x
for x in set(
(
best_hps[hp]["optimiser"],
best_hps[hp]["l_rate"],
best_hps[hp]["epsilon"],
)
for hp in hp_search_methods
)
]
test_params = {
"opt_name": [hp[0] for hp in hp_matrix],
"l_rate": [hp[1] for hp in hp_matrix],
"epsilon": [hp[2] for hp in hp_matrix],
}
# parameters that differ from the default values but
# that we want to keep constant through each iteration
control_params = {
"model_name": model_name,
}
compare_configs(
test_params,
model_name,
train_ds,
val_ds,
test_ds,
control_params,
)
14/14 [==============================] - 1s 53ms/step - loss: 10.8099 - f1_score: 0.7993 - accuracy: 0.7410 - precision_1: 0.8333 - recall_1: 0.7679 3/3 [==============================] - 0s 54ms/step - loss: 17.0698 - f1_score: 0.6325 - accuracy: 0.5169 - precision_1: 0.6491 - recall_1: 0.6167 3/3 [==============================] - 0s 56ms/step - loss: 19.9935 - f1_score: 0.6218 - accuracy: 0.5055 - precision_1: 0.6271 - recall_1: 0.6167 14/14 [==============================] - 1s 53ms/step 3/3 [==============================] - 0s 52ms/step 3/3 [==============================] - 0s 55ms/step
14/14 [==============================] - 1s 53ms/step - loss: 104.0018 - f1_score: 0.8034 - accuracy: 0.6715 - precision_2: 0.6715 - recall_2: 1.0000 3/3 [==============================] - 0s 55ms/step - loss: 104.1037 - f1_score: 0.8054 - accuracy: 0.6742 - precision_2: 0.6742 - recall_2: 1.0000 3/3 [==============================] - 0s 55ms/step - loss: 107.7250 - f1_score: 0.7947 - accuracy: 0.6593 - precision_2: 0.6593 - recall_2: 1.0000 14/14 [==============================] - 1s 56ms/step 3/3 [==============================] - 0s 50ms/step 3/3 [==============================] - 0s 50ms/step
best_optimiser = "sgd"
best_l_rate = 0.5362420800000001
best_epsilon = 0.05256905
# parameters that we want to iterate through
test_params = {
"epochs": [50]
}
# parameters that differ from the default values but
# that we want to keep constant through each iteration
# of the test parameters
control_params = {
"model_name": model_name,
"opt_name": best_optimiser,
"l_rate": best_l_rate,
"epsilon": best_l_rate
}
compare_configs(
test_params,
model_name,
train_ds,
val_ds,
test_ds,
control_params,
)
14/14 [==============================] - 1s 52ms/step - loss: 167.7011 - f1_score: 0.8034 - accuracy: 0.6715 - precision_3: 0.6715 - recall_3: 1.0000 3/3 [==============================] - 0s 52ms/step - loss: 167.3920 - f1_score: 0.8054 - accuracy: 0.6742 - precision_3: 0.6742 - recall_3: 1.0000 3/3 [==============================] - 0s 54ms/step - loss: 173.4474 - f1_score: 0.7947 - accuracy: 0.6593 - precision_3: 0.6593 - recall_3: 1.0000 14/14 [==============================] - 1s 52ms/step 3/3 [==============================] - 0s 49ms/step 3/3 [==============================] - 0s 51ms/step
Ideal epoch number: 30
model_hps[model_name] = {
"epochs": 30,
"l_rate": best_l_rate,
"epsilon": best_epsilon,
"optimiser": best_optimiser
}
model_hps
{'perceptron': {'epochs': 20,
'l_rate': 0.022246600000000002,
'epsilon': 0.7611846600000001,
'optimiser': 'adagrad'},
'test_cnn_1': {'epochs': 15,
'l_rate': 0.50438527,
'epsilon': 0.05359655,
'optimiser': 'adam'},
'test_cnn_2': {'epochs': 30,
'l_rate': 0.5362420800000001,
'epsilon': 0.05256905,
'optimiser': 'sgd'}}
model_name = "test_cnn_3"
best_hps = {
method: get_best_hyperparams(model_name, train_ds, val_ds, optimiser_range, method)
for method in hp_search_methods
}
best_hps
Trial 5 Complete [00h 00m 24s] val_accuracy: 0.6741573214530945 Best val_accuracy So Far: 0.7191011309623718 Total elapsed time: 00h 02m 01s INFO:tensorflow:Oracle triggered exit
{'random': {'optimiser': 'nadam', 'l_rate': 0.15811354, 'epsilon': 0.06356881},
'bayesian': {'optimiser': 'adam',
'l_rate': 0.29507792,
'epsilon': 0.35725662}}
# parameters that we want to iterate through
hp_matrix = [
x
for x in set(
(
best_hps[hp]["optimiser"],
best_hps[hp]["l_rate"],
best_hps[hp]["epsilon"],
)
for hp in hp_search_methods
)
]
test_params = {
"opt_name": [hp[0] for hp in hp_matrix],
"l_rate": [hp[1] for hp in hp_matrix],
"epsilon": [hp[2] for hp in hp_matrix],
}
# parameters that differ from the default values but
# that we want to keep constant through each iteration
control_params = {
"model_name": model_name,
}
compare_configs(
test_params,
model_name,
train_ds,
val_ds,
test_ds,
control_params,
)
14/14 [==============================] - 1s 61ms/step - loss: 67.6354 - f1_score: 0.8092 - accuracy: 0.6835 - precision_1: 0.6796 - recall_1: 1.0000 3/3 [==============================] - 0s 61ms/step - loss: 71.3541 - f1_score: 0.8054 - accuracy: 0.6742 - precision_1: 0.6742 - recall_1: 1.0000 3/3 [==============================] - 0s 62ms/step - loss: 75.6457 - f1_score: 0.7947 - accuracy: 0.6593 - precision_1: 0.6593 - recall_1: 1.0000 14/14 [==============================] - 1s 58ms/step 3/3 [==============================] - 0s 56ms/step 3/3 [==============================] - 0s 57ms/step
14/14 [==============================] - 1s 61ms/step - loss: 7.5362 - f1_score: 0.7269 - accuracy: 0.6379 - precision_2: 0.7363 - recall_2: 0.7179 3/3 [==============================] - 0s 62ms/step - loss: 9.4470 - f1_score: 0.6977 - accuracy: 0.5618 - precision_2: 0.6522 - recall_2: 0.7500 3/3 [==============================] - 0s 63ms/step - loss: 10.2503 - f1_score: 0.6230 - accuracy: 0.4945 - precision_2: 0.6129 - recall_2: 0.6333 14/14 [==============================] - 1s 59ms/step 3/3 [==============================] - 0s 60ms/step 3/3 [==============================] - 0s 60ms/step
best_optimiser = "nadam"
best_l_rate = 0.15811354
best_epsilon = 0.06356881
# parameters that we want to iterate through
test_params = {
"epochs": [50]
}
# parameters that differ from the default values but
# that we want to keep constant through each iteration
# of the test parameters
control_params = {
"model_name": model_name,
"opt_name": best_optimiser,
"l_rate": best_l_rate,
"epsilon": best_l_rate
}
compare_configs(
test_params,
model_name,
train_ds,
val_ds,
test_ds,
control_params,
)
14/14 [==============================] - 1s 61ms/step - loss: 114.2310 - f1_score: 0.0000e+00 - accuracy: 0.3285 - precision_8: 0.0000e+00 - recall_8: 0.0000e+00 3/3 [==============================] - 0s 63ms/step - loss: 118.4614 - f1_score: 0.0000e+00 - accuracy: 0.3258 - precision_8: 0.0000e+00 - recall_8: 0.0000e+00 3/3 [==============================] - 0s 61ms/step - loss: 116.9338 - f1_score: 0.0000e+00 - accuracy: 0.3407 - precision_8: 0.0000e+00 - recall_8: 0.0000e+00 14/14 [==============================] - 1s 59ms/step 3/3 [==============================] - 0s 60ms/step 3/3 [==============================] - 0s 60ms/step
Ideal epoch number: 10
model_hps[model_name] = {
"epochs": 10,
"l_rate": best_l_rate,
"epsilon": best_epsilon,
"optimiser": best_optimiser
}
model_hps
{'perceptron': {'epochs': 20,
'l_rate': 0.022246600000000002,
'epsilon': 0.7611846600000001,
'optimiser': 'adagrad'},
'test_cnn_1': {'epochs': 15,
'l_rate': 0.50438527,
'epsilon': 0.05359655,
'optimiser': 'adam'},
'test_cnn_2': {'epochs': 30,
'l_rate': 0.5362420800000001,
'epsilon': 0.05256905,
'optimiser': 'sgd'},
'test_cnn_3': {'epochs': 10,
'l_rate': 0.15811354,
'epsilon': 0.06356881,
'optimiser': 'nadam'}}
# parameters that we want to iterate through
keys = list(model_hps.keys())
test_params = {
"model_name": keys,
"opt_name": [model_hps[k]["optimiser"] for k in keys],
"l_rate": [model_hps[k]["l_rate"] for k in keys],
"epsilon": [model_hps[k]["epsilon"] for k in keys],
"epochs": [model_hps[k]["epochs"] for k in keys],
}
# parameters that differ from the default values but
# that we want to keep constant through each iteration
control_params = {}
compare_configs(
test_params,
model_name,
train_ds,
val_ds,
test_ds,
control_params,
)
14/14 [==============================] - 0s 5ms/step - loss: 0.5380 - f1_score: 0.8976 - accuracy: 0.8681 - precision_9: 0.9377 - recall_9: 0.8607 3/3 [==============================] - 0s 7ms/step - loss: 2.8787 - f1_score: 0.6726 - accuracy: 0.5843 - precision_9: 0.7170 - recall_9: 0.6333 3/3 [==============================] - 0s 7ms/step - loss: 3.3214 - f1_score: 0.6833 - accuracy: 0.5824 - precision_9: 0.6833 - recall_9: 0.6833 14/14 [==============================] - 0s 5ms/step 3/3 [==============================] - 0s 5ms/step 3/3 [==============================] - 0s 5ms/step
14/14 [==============================] - 1s 40ms/step - loss: 294.8774 - f1_score: 0.8834 - accuracy: 0.8537 - precision_10: 0.9506 - recall_10: 0.8250 3/3 [==============================] - 0s 42ms/step - loss: 1037.4180 - f1_score: 0.6372 - accuracy: 0.5393 - precision_10: 0.6792 - recall_10: 0.6000 3/3 [==============================] - 0s 43ms/step - loss: 1015.6523 - f1_score: 0.6724 - accuracy: 0.5824 - precision_10: 0.6964 - recall_10: 0.6500 14/14 [==============================] - 1s 39ms/step 3/3 [==============================] - 0s 38ms/step 3/3 [==============================] - 0s 39ms/step
14/14 [==============================] - 1s 54ms/step - loss: 64.5133 - f1_score: 0.4167 - accuracy: 0.4964 - precision_11: 0.9375 - recall_11: 0.2679 3/3 [==============================] - 0s 55ms/step - loss: 72.7705 - f1_score: 0.3467 - accuracy: 0.4494 - precision_11: 0.8667 - recall_11: 0.2167 3/3 [==============================] - 0s 55ms/step - loss: 73.6770 - f1_score: 0.2162 - accuracy: 0.3626 - precision_11: 0.5714 - recall_11: 0.1333 14/14 [==============================] - 1s 57ms/step 3/3 [==============================] - 0s 51ms/step 3/3 [==============================] - 0s 52ms/step
14/14 [==============================] - 1s 62ms/step - loss: 33.9206 - f1_score: 0.8070 - accuracy: 0.6811 - precision_12: 0.6797 - recall_12: 0.9929 3/3 [==============================] - 0s 71ms/step - loss: 35.1535 - f1_score: 0.7891 - accuracy: 0.6517 - precision_12: 0.6667 - recall_12: 0.9667 3/3 [==============================] - 0s 61ms/step - loss: 34.4937 - f1_score: 0.7919 - accuracy: 0.6593 - precision_12: 0.6629 - recall_12: 0.9833 14/14 [==============================] - 1s 57ms/step 3/3 [==============================] - 0s 55ms/step 3/3 [==============================] - 0s 59ms/step
Whilst the perceptron and single convolutional layer models display a similar level of performance, the 3-layer convolutional network has the clear edge both in terms of accuracy and F1 score.